import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

/**
 * create by jsm on 2018/9/21
 */
public class Wechat implements Task, PageProcessor {
    private List<String> urlList=new ArrayList <>();
    private Site site = Site.me().setCycleRetryTimes(5).setRetryTimes(5).setSleepTime(5000).setTimeOut(3 * 60 * 1000)
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
            .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
            .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
            .addHeader("Cookie","CXID=5845682A8C4033ACDE8BFFD6024C0790; SUID=4ABEEC735B68860A5A827C85000E9800; SUV=00724ADF73ECBE4A5AB4AAA8EA3F2302; SMYUV=1525312364775010; usid=-2E6javxnTmhSm26; wuid=AAEa2PtNIQAAAAqROlQCQwUAAAA=; UM_distinctid=1649950129d8e6-0def43dd7cccb6-163d6955-13c680-1649950129e197; ad=Zyllllllll2zkXhVlllllVmA3MYllllltMhMvkllllUlllll9Cxlw@@@@@@@@@@@; ABTEST=0|1537518865|v1; weixinIndexVisited=1; JSESSIONID=aaaKIGcFrn31n6nHj9Fvw; ppinf=5|1537522292|1538731892|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0NTolRTYlOTElQjglRTQlQkQlQTAlRTUlQjAlOEYlRTglODIlQTUlRTglODQlQjh8Y3J0OjEwOjE1Mzc1MjIyOTJ8cmVmbmljazo0NTolRTYlOTElQjglRTQlQkQlQTAlRTUlQjAlOEYlRTglODIlQTUlRTglODQlQjh8dXNlcmlkOjQ0Om85dDJsdURCMWpJNTRPTnJvejlQYXRiNWhLLTRAd2VpeGluLnNvaHUuY29tfA; pprdig=Otxazlk1y2TV0W8Diy-031MAfE4b6eXFqjVOITUnwdaVDFujiwFzQJV2ye6ffVGjcm3z4gyy_JHnsp2lGq9KBAbJ_-s37wZBHWtuwpxa-xlpIBcOSPnEgU4O5wuq-ST7GUzcUPzehIvgDgN--7IHN0MnLT5SxVR3oZj-eR0pXmE; sgid=03-27978503-AVukunRwyMibQBialIV5BK8AM; PHPSESSID=oeg6qni0s9fk0i43ig1t3r7df2; SUIR=AFB4E87A0005775D83B6BEA901E36209; SNUID=9429F2964A4C3CE2317463394ADC3B1E; IPLOC=CN3301; ppmdig=15378516790000001a17342eef929b6a71fc8f793291b124; sct=4\n" +
                    "Host: weixin.sogou.com\n" +
                    "Referer: http://weixin.sogou.com/weixin?query=%E5%AF%9D%E5%AE%A4%E6%94%B9%E9%80%A0&hp=40&sut=1534&lkt=1%2C1537851684501%2C1537851684501&_sug_=y&sst0=1537851684605&oq=&stj0=0&stj1=4&stj=0%3B4%3B0%3B0&stj2=0&hp1=&_sug_type_=&s_from=input&ri=0&type=2&page=10&ie=utf8&w=01015002&dr=1");


    @Override
    public String getUUID() {
        return UUID.randomUUID().toString();
    }

    public List <String> getUrlList() {
        return urlList;
    }

    public void setUrlList(List <String> urlList) {
        this.urlList = urlList;
    }

    @Override
    public void process(Page page) {
        System.out.println(page.getHtml());
        List <String> items = page.getHtml().xpath("//h3/a[@target='_blank']/@href").all();

        for (String item : items) {
            if (item.endsWith("&new=1")) {
                urlList.add(item);
            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Wechat wechat = new Wechat();
        for (int i = 1; i < 30; i++) {
            Spider.create(wechat).addUrl("http://weixin.sogou.com/weixin?type=2&s_from=input&query=%E5%AF%9D%E5%AE%A4%E6%94%B9%E9%80%A0&page=" + i).
                    // addPipeline(new FilePipeline("D:\\webmagic\\")).
                            addPipeline(new ConsolePipeline()).thread(1).run();
        }
        File file = new File("/Users/yimouser/Desktop/qsgz.txt");
        List <String> urlList = wechat.getUrlList();
        try {
            BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(file));
            for (String url : urlList) {
                bufferedWriter.write(url);
                bufferedWriter.newLine();
                bufferedWriter.newLine();
            }
            bufferedWriter.flush();
            bufferedWriter.close();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
